{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50.5\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "from lxml import etree\n",
    "from functools import reduce\n",
    "path='/home/quanwei/workspaceFolder/code-python/gpa.html'\n",
    "c=[]\n",
    "with open(path,'r') as f:\n",
    "    text=f.read()\n",
    "    # print(text)\n",
    "    html=etree.HTML(text)\n",
    "    trs= html.xpath(\"//tr\")\n",
    "    for i in trs:\n",
    "      tds=i.xpath(\"./td/text()\")\n",
    "      c.append(tds)\n",
    "      # for j in tds :\n",
    "      #    print(j,end=', ')\n",
    "      # print()\n",
    "c=filter(lambda x:x[2].startwith('必') ,c)\n",
    "up=0\n",
    "down=reduce(lambda x,y:x[3]+y[3],c)\n",
    "print(down)\n",
    "for i in c:\n",
    "  print(i)\n",
    "  if i[4].isnumeric():\n",
    "    up+=i[3]*i[4]\n",
    "  elif i[4]=='优秀':\n",
    "    up+=i[3]*90\n",
    "  else:\n",
    "    up+=i[3]*80\n",
    "print(up)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'lxml'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mlxml\u001b[39;00m \u001b[39mimport\u001b[39;00m etree\n\u001b[1;32m      2\u001b[0m text \u001b[39m=\u001b[39m \u001b[39m'''\u001b[39m\n\u001b[1;32m      3\u001b[0m \u001b[39m<div>\u001b[39m\n\u001b[1;32m      4\u001b[0m \u001b[39m    <ul>\u001b[39m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[39m </div>\u001b[39m\n\u001b[1;32m     12\u001b[0m \u001b[39m'''\u001b[39m\n\u001b[1;32m     13\u001b[0m html \u001b[39m=\u001b[39m etree\u001b[39m.\u001b[39mHTML(text)\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'lxml'"
     ]
    }
   ],
   "source": [
    "from lxml import etree\n",
    "text = '''\n",
    "<div>\n",
    "    <ul>\n",
    "         <li class=\"item-0\"><a href=\"https://ask.hellobi.com/link1.html\">first item</a></li>\n",
    "         <li class=\"item-1\"><a href=\"https://ask.hellobi.com/link2.html\">second item</a></li>\n",
    "         <li class=\"item-inactive\"><a href=\"https://ask.hellobi.com/link3.html\">third item</a></li>\n",
    "         <li class=\"item-1\"><a href=\"https://ask.hellobi.com/link4.html\">fourth item</a></li>\n",
    "         <li class=\"item-0\"><a href=\"https://ask.hellobi.com/link5.html\">fifth item</a>\n",
    "     </ul>\n",
    " </div>\n",
    "'''\n",
    "html = etree.HTML(text)\n",
    "result = etree.tostring(html)\n",
    "# print(result.decode('utf-8'))\n",
    "\n",
    "# @属性名\n",
    "print(\"属性：\",html.xpath(\"//li/@class\"))\n",
    "\n",
    "# text()\n",
    "print(\"内容：\",html.xpath(\"//li[@class=\\\"item-0\\\"]/a/text()\"))\n",
    "\n",
    "print(\"内容：\",html.xpath(\"//a/text()\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 导入所需的库\n",
    "import urllib.request as req\n",
    "from lxml import etree\n",
    "\n",
    "# 待爬取网页的url\n",
    "url='https://top.baidu.com/board?tab=realtime&sa=fyb_realtime_31065'\n",
    "\n",
    "# 爬取页面信息\n",
    "ret = req.urlopen(url)\n",
    "html = etree.HTML(ret.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 从页面信息提取：标题、热搜指数、图片、链接\n",
    "ctx = ''\n",
    "for i in range(1, 31):\n",
    "    title = html.xpath(\n",
    "        f'/html/body/div/div/main/div[2]/div/div[2]/div[{i}]/div[2]/a/div[1]/text()')\n",
    "    index = html.xpath(\n",
    "        f'/html/body/div/div/main/div[2]/div/div[2]/div[{i}]/div[1]/div[2]/text()')\n",
    "    img = html.xpath(\n",
    "        f'/html/body/div/div/main/div[2]/div/div[2]/div[{i}]/a/img/@src')\n",
    "    link = html.xpath(\n",
    "        f'/html/body/div/div/main/div[2]/div/div[2]/div[{i}]/div[2]/a/@href')\n",
    "\n",
    "    ctx += f'''\n",
    "    <div class=\"my-4 col-lg-3 mx-auto\">\n",
    "    <h2 class=\"card-header mx-auto my-2\">{title[0].strip()}</h2>\n",
    "    <p>热搜指数：{index[0].strip()}</p>\n",
    "    <div class=\"card mx-auto\">\n",
    "    <img src='{img[0].strip()}'>\n",
    "    <div class=\"card-body\">\n",
    "    <a href=\"{link[0].strip()}\" class=\"btn btn-primary end\">了解更多</a>\n",
    "    </div></div></div>\n",
    "    '''\n",
    "\n",
    "# 要展示的页面\n",
    "res = f'''\n",
    "<html>\n",
    "<head>\n",
    "<title>hot</title>\n",
    "<link href=\"https://cdn.jsdelivr.net/npm/bootstrap@5.2.1/dist/css/bootstrap.min.css\" \n",
    "      rel=\"stylesheet\"\n",
    "      integrity=\"sha384-iYQeCzEYFbKjA/T2uDLTpkwGzCiq6soy8tYaI1GyVh/UjpbCx/TYkiZhlZB6+fzT\" \n",
    "      crossorigin=\"anonymous\">\n",
    "</head>\n",
    "<body>\n",
    "<div class=\"container row mx-auto\">\n",
    "{ctx}\n",
    "</div>\n",
    "</body>\n",
    "</html>\n",
    "'''\n",
    "\n",
    "# 写入文件\n",
    "with open('hot.html', 'w') as f:\n",
    "    f.write(res)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib.parse\n",
    "import urllib.request\n",
    "import json\n",
    "\n",
    "url = 'https://movie.douban.com/j/search_subjects?type=movie&tag=%E7%83%AD%E9%97%A8&page_limit=50&page_start=0'\n",
    "# 豆瓣最新 热门\n",
    "\n",
    "herders = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36',\n",
    "           'Referer': 'https://movie.douban.com', 'Connection': 'keep-alive'}\n",
    "# 请求头信息\n",
    "\n",
    "req = urllib.request.Request(url, headers=herders)\n",
    "# 设置请求头\n",
    "response = urllib.request.urlopen(req)\n",
    "# 发起请求，得到response响应\n",
    "\n",
    "hjson = json.loads(response.read())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "jsonFile = json.dumps(hjson, ensure_ascii=False)\n",
    "with open(\"../豆瓣.json\", 'w') as f:\n",
    "    f.write(jsonFile)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mRunning cells with 'base' requires the ipykernel package.\n",
      "\u001b[1;31mRun the following command to install 'ipykernel' into the Python environment. \n",
      "\u001b[1;31mCommand: 'conda install -n base ipykernel --update-deps --force-reinstall'"
     ]
    }
   ],
   "source": [
    "type(hjson)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(jsonFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_ls = hjson[\"subjects\"]\n",
    "print(type(sub_ls))\n",
    "print(type(sub_ls[0]))\n",
    "sub_ls2 = sorted(sub_ls, key=lambda x: x[\"rate\"],reverse=True)\n",
    "# hjson2 = json.loads(dict({\"subjects\": sub_ls2}))\n",
    "json_str=json.dumps({\"subjects\": sub_ls2},ensure_ascii=False)\n",
    "\n",
    "with open(\"../豆瓣.json\", 'w') as f:\n",
    "    f.write(json_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# json转换为字典\n",
    "for item in hjson[\"subjects\"]:\n",
    "    print(item[\"rate\"],item[\"title\"])\n",
    "    # 打印每条电影的评分与标题"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# proj\n",
    "\n",
    "import requests\n",
    "import json\n",
    "\n",
    "\n",
    "# from fake_useragent import UserAgent\n",
    "headers = {\n",
    "    \"Accept\": \"application/json, text/javascript, */*; q=0.01\",\n",
    "    \"Accept-Encoding\": \"gzip, deflate, br\",\n",
    "    \"Accept-Language\": \"en-US,en;q=0.5\",\n",
    "    \"Cache-Control\": \"no-cache\",\n",
    "    \"Connection\": \"keep-alive\",\n",
    "    \"Content-Length\": \"79\",\n",
    "    \"Content-Type\": \"application/x-www-form-urlencoded; charset=UTF-8\",\n",
    "    # \"Cookie\": \"JSESSIONID=408BC6B2C5CDD6635D54D3DC82538502; http_waf_cookie=47f0b506-7472-472c207521b4b83bcac1fed1e2181ec8a7a3; https_waf_cookie=5bd5eec7-7d21-4dcaf170d18c6feba874311f2bf5ceb598d6; HWWAFSESID=fb46a73ca79f6bdc29; HWWAFSESTIME=1669964395754; aisteUv=1669964329470511606286; aisiteJsSessionId=16699643294702262018437\",\n",
    "    \"Host\": \"www.sara.gov.cn\",\n",
    "    \"Origin\": \"https://www.sara.gov.cn\",\n",
    "    \"Pragma\": \"no-cache\",\n",
    "    \"Referer\": \"https://www.sara.gov.cn/gjzjswj/zjjcxxcxxt/zjhdcsjbxx/index.shtml\",\n",
    "    \"request-by\": \"ajax-request-tag\",\n",
    "    \"Sec-Fetch-Dest\": \"empty\",\n",
    "    \"Sec-Fetch-Mode\": \"cors\",\n",
    "    \"Sec-Fetch-Site\": \"same-origin\",\n",
    "    \"User-Agent\": \"Mozilla/5.0 (X11; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0\",\n",
    "    \"X-Requested-With\": \"XMLHttpRequest\"\n",
    "}\n",
    "\n",
    "\n",
    "url = \"https://www.sara.gov.cn/eportal/ui?portal.url=/portlet/place-info-front!queryList.portlet&moduleId=b68f71decbfb4ff4b274cac2bf67204b&pageId=614c812b4af64b4e91e31a7a7bc9f12a\"\n",
    "# for pagenum in range(1, 100):\n",
    "data = {\n",
    "    \"province\": \"\",\n",
    "    \"city\": \"\",\n",
    "    \"town\": \"\",\n",
    "    \"faction\": \"\",\n",
    "    \"religionName\": \"\",\n",
    "    \"keyWord\": \"\",\n",
    "    \"currentPage\": \"1\",\n",
    "    \"pageSize\": \"15\"\n",
    "}\n",
    "\n",
    "file={\"val\":\"\"}\n",
    "text=\"\"\n",
    "for i in range(1,10):\n",
    "    data[\"currentPage\"]=str(i)\n",
    "    resp = requests.post(url=url, headers=headers, data=data)\n",
    "    text = resp.content.decode()\n",
    "    file[\"val\"]=file[\"val\"]+text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"../fj.json\",'w') as f:\n",
    "  jf=json.dumps(file,ensure_ascii=False).replace('\\n',' ')\n",
    "  f.write(jf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lxml import etree\n",
    "from functools import reduce\n",
    "path = '/home/quanwei/workspaceFolder/code-python/gpa.html'\n",
    "c = []\n",
    "with open(path, 'r') as f:\n",
    "    text = f.read()\n",
    "    html = etree.HTML(text)\n",
    "    trs = html.xpath(\"//tr\")\n",
    "    for i in trs:\n",
    "        tds = i.xpath(\"./td/text()\")\n",
    "        c.append(tds)\n",
    "c = filter(lambda x: str(x[2]).startswith('必'), c)\n",
    "up = 0\n",
    "down = 0\n",
    "for i in c:\n",
    "    num = float(i[3])\n",
    "    down += num\n",
    "    print(i)\n",
    "    p = 0\n",
    "    if i[4].isnumeric():\n",
    "        x = float(i[4])\n",
    "        if x >= 90:\n",
    "            p = 5\n",
    "        elif x >= 80:\n",
    "            p = 4\n",
    "        elif x >= 70:\n",
    "            p = 3\n",
    "    elif i[4] == '优秀':\n",
    "        p = 5\n",
    "    elif i[4] == '良好':\n",
    "        p = 4\n",
    "    print(p)\n",
    "    up += num*p\n",
    "print(f'{up}/{down}={up/down:.2f}')\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "8a5edab282632443219e051e4ade2d1d5bbc671c781051bf1437897cbdfea0f1"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
